#!/bin/bash
# This is to check puppet service
# if puppet config to run then check puppet service
echo "###############################"
date
/sbin/chkconfig puppet
if [ $? -eq 0 ]
then
  #if puppet no run then run it
  /sbin/service puppet status
  if [ $?  -ne 0 ]
  then
    killall -9 puppet
    killall -9 puppet
    /sbin/service puppet restart
  fi
  tail -n 50 /var/log/messages |grep puppet  |grep "Could not run"
  if [ $? -eq 0 ]
  then
    killall -9 puppet
    killall -9 puppet
    /sbin/service puppet restart
  fi
  # delete puppet process that running over 4 hours
  /bin/ps -ef |grep "puppet agent: applying configurat" |grep -v grep > /dev/null
  if [ $? == 0 ]
  then
          pid=`/bin/ps -ef |grep "puppet agent: applying configurat" |grep -v grep|awk '{print $2}'`
          times=`/bin/ps  -o etime= $pid | tr '-' ':' | awk -F: '{ total=0; m=1; } { for (i=0; i < NF; i++) {total += $(NF-i)*m; m *= i >= 2 ? 24 : 60 }} {print total}'`
          if [ "$times" -ge "14400" ]
          then
                  kill -9 $pid
                  killall -9 puppet
                  killall -9 puppet
                  /etc/init.d/puppet restart
          fi
  fi
  # to avoid puppet can not reload it self
  #/opt/puppetlabs/puppet/cache/state/agent_catalog_run.lock #This file lock
  if [ -f /opt/puppetlabs/puppet/cache/state/last_run_summary.yaml ]
  then
    find /opt/puppetlabs/puppet/cache/state/last_run_summary.yaml -mmin +120   -exec service puppet restart \;
  fi
fi



# another function to check ntp
/sbin/service ntpd status
  if [ $?  -ne 0 ]
  then
    /usr/sbin/ntpdate ntp1.ihep.ac.cn
    /usr/sbin/hwclock -w
    /sbin/service ntpd restart
    /sbin/service puppet restart
  fi
